package com.saibo.spider.gather;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.saibo.spider.commons.SubmitRedisCommons;
import com.saibo.spider.pojo.SinaUserInfo;
import com.saibo.spider.pojo.SinaWeibo;
import com.saibo.spider.utils.DateUtils;
import com.saibo.spider.utils.FileUtil;
import com.saibo.spider.utils.HttpRequestUtil;
import com.saibo.spider.utils.HttpResponseUtil;
import com.saibo.spider.utils.MD5Util;
import com.saibo.spider.utils.SinaBlogScriptUtil;
import com.saibo.spider.utils.StringUtil;

/**
 * 根据微博用户主页的URL以及要采集博文的日期获取当日发文，以及更新用户的基本信息。
 * 
 * @author TerransForce
 *
 */
public class MouldSinaUserBlog {

	Logger log = Logger.getLogger(MouldSinaUserBlog.class);
	SinaUserInfo userinfo = new SinaUserInfo();// 存用户
	List<SinaWeibo> blogList = new ArrayList<SinaWeibo>();// 存发文

	public static void main(String[] args) throws InterruptedException {

		// 孙艺洲 http://weibo.com/sunyizhou2008
		// http://weibo.com/realangelababy
		// http://weibo.com/bobog
		// http://weibo.com/0626boboyu
		// http://weibo.com/u/1737350632
		// http://weibo.com/qihang617
		// http://weibo.com/dengsha
		// http://weibo.com/chenghaofeng
		// http://weibo.com/u/1689055184
		// http://weibo.com/u/1630903355
		// http://weibo.com/gufangbuzishang001
		// http://weibo.com/zhonghanliang
		// http://weibo.com/u/6084440520

		MouldSinaUserBlog msub = new MouldSinaUserBlog();
		String html = msub.stitchUrl("http://weibo.com/bobog", DateUtils.getDateStr_YMD(-1));
		msub.parseHtml(html);
		System.err.println(StringUtil.objectToJson(msub.userinfo));
		List<SinaUserInfo> userlist = new ArrayList<SinaUserInfo>();
		if (!StringUtil.isNullOrEmpty(msub.userinfo.getNick_name())) {
			msub.userinfo.setCrawlerDate(System.currentTimeMillis() + "");
			userlist.add(msub.userinfo);
			SubmitRedisCommons.submitRedisData("sina_user_info", StringUtil.objectToJson(userlist));
			SubmitRedisCommons.submitRedisData("sina_blog_info", StringUtil.objectToJson(msub.blogList));
		}
		/*
		 * MouldSinaUserBlog msub = new MouldSinaUserBlog(); String html =
		 * msub.stitchUrl("http://weibo.com/sunyizhou2008", "2017-01-01");
		 * msub.parseHtml(html);
		 * System.err.println(StringUtil.objectToJson(msub.userinfo));
		 * List<SinaUserInfo> userlist = new ArrayList<SinaUserInfo>();
		 * if(!StringUtil.isNullOrEmpty(msub.userinfo.getNick_name())){
		 * msub.userinfo.setCrawlerDate(System.currentTimeMillis()+"" );
		 * userlist.add(msub.userinfo);
		 * SubmitRedisCommons.submitRedisData("sina_user_info",
		 * StringUtil.objectToJson(userlist));
		 * SubmitRedisCommons.submitRedisData("sina_blog_info",
		 * StringUtil.objectToJson(msub.blogList)); }
		 */
	}

	/**
	 * 请求微博的URL返回html代码
	 */
	public String stitchUrl(String url, String datetime) {
		userinfo.setUrl(url);
		userinfo.setRowkey(MD5Util.GET16BITMD5(url));
		url = url + "?is_ori=1&is_forward=1&is_text=1&is_pic=1&is_video=1&is_music=1&is_article=1&key_word=&start_time="
				+ datetime + "&end_time=" + datetime + "&is_search=1&is_searchadv=1";
		System.out.println(url);
		CloseableHttpResponse httpResponse = HttpRequestUtil.getRequestMethod(url, requestHeader());
		System.out.println(httpResponse.getStatusLine());
		String html = HttpResponseUtil.getHtml(httpResponse);
		return html;
	}

	/**
	 * 解析新浪微博页面,分别解析用户的传播指数,
	 */
	public void parseHtml(String html) {
		if (StringUtil.isNullOrEmpty(html))
			return;
		Document document = Jsoup.parse(html);
		List<String> scriptList = SinaBlogScriptUtil.replaceScript(document.select("script"));
		for (String string : scriptList) {
			Map<String, Object> jsonMap = StringUtil.jsonToMap(string);
			// 获取发文的Script标签
			if ((jsonMap.get("ns").toString()).equals("pl.content.homeFeed.index")) {
				// 获取发文
				if (jsonMap.get("domid").toString().startsWith("Pl_Official_MyProfileFeed")) {
					String articleHtml = jsonMap.get("html").toString();
					parseUserArticle(articleHtml);
				}
				// 获取用户属性
				if (jsonMap.get("domid").toString().startsWith("Pl_Core_UserInfo")) {
					String userinfoHtml = jsonMap.get("html").toString();
					parseUserABInfo(userinfoHtml);
				}
			} else if ("".equals(jsonMap.get("ns").toString())) {
				if ((jsonMap.get("domid").toString()).startsWith("Pl_Core_T8CustomTriColumn")) {
					String userIHtml = jsonMap.get("html").toString();
					parseUserFansInfo(userIHtml);
				}
			} else if ("pl.header.head.index".equals(jsonMap.get("ns").toString())) {
				// 获取用户的头像信息
				if ((jsonMap.get("domid").toString()).startsWith("Pl_Official_Header")) {
					String headerHtml = jsonMap.get("html").toString();
					parseUserHeaderInfo(headerHtml);
				}
			}
		}
	}

	/**
	 * 解析用户发文信息
	 * 
	 * @param html
	 */
	public void parseUserArticle(String html) {
		Document document_article_html = Jsoup.parse(html);
		Elements articleElements = document_article_html.select("div.WB_feed>div[action-type=feed_list_item]");
		System.out.println("采集日期文章发文数:" + articleElements.size());
		for (Element element : articleElements) {
			SinaWeibo weibo = new SinaWeibo();
			Element listElementOne = element.select("div").first();
			String mid = listElementOne.attr("mid");// 微博文章的MID
			Element dateAndPlatElement = element.select("div>div.WB_feed_detail>div.WB_detail>div.WB_from").first();
			String articleDatetime = dateAndPlatElement.select("div.WB_from>a").first().attr("date");
			String articleUrl = dateAndPlatElement.select("div.WB_from>a").first().attr("href");
			String platform = "";
			if (dateAndPlatElement.select("div.WB_from>a").size() > 1) {
				platform = StringUtil.element2text(dateAndPlatElement.select("div.WB_from>a").get(1));
			}
			Element contentElement = element.select("div>div.WB_feed_detail>div.WB_detail>div.WB_text").first();
			String content = StringUtil.element2text(contentElement);
			// 获取阅读点赞
			int forward_count = 0;
			int comment_count = 0;
			int like_count = 0;
			// 转发数
			Element forwardElement = listElementOne
					.select("div>div.WB_feed_handle>div.WB_handle>ul>li>a[action-type=fl_forward]").first();
			String forwardStr = StringUtil.element2text(forwardElement);
			forwardStr = forwardStr.replaceAll("", "").replaceAll(" ", "").replaceAll("ñ", "");
			if (!StringUtil.isNullOrEmpty(forwardStr)) {
				forward_count = getInteger(forwardStr);
			}
			// 回复数
			Element commentElement = listElementOne
					.select("div>div.WB_feed_handle>div.WB_handle>ul>li>a[action-type=fl_comment]").first();
			String commentStr = StringUtil.element2text(commentElement);
			commentStr = commentStr.replaceAll("", "").replaceAll(" ", "").replaceAll("ñ", "");
			if (!StringUtil.isNullOrEmpty(commentStr)) {
				comment_count = getInteger(commentStr);
			}
			// 点赞数
			Element likeElement = listElementOne
					.select("div>div.WB_feed_handle>div.WB_handle>ul>li>a[action-type=fl_like]").first();
			String likeStr = StringUtil.element2text(likeElement);
			likeStr = likeStr.replaceAll("", "").replaceAll(" ", "").replaceAll("ñ", "");
			if (!StringUtil.isNullOrEmpty(likeStr)) {
				like_count = getInteger(likeStr);
			}
			articleUrl = "http://weibo.com/" + articleUrl;
			weibo.setRowkey(MD5Util.GET16BITMD5(filterUrl(articleUrl)));
			weibo.setUrl(filterUrl(articleUrl));
			weibo.setMid_(mid);
			weibo.setCreate_time(articleDatetime);
			weibo.setContent(content);
			weibo.setPlatform(platform);
			weibo.setAuthor_rowkey(userinfo.getRowkey());
			weibo.setIsforward("");
			weibo.setSource_rowkey("");
			weibo.setForward_count(forward_count + "");
			weibo.setComment_count(comment_count + "");
			weibo.setLike_count(like_count + "");
			weibo.setCrawler_time(System.currentTimeMillis() + "");
			weibo.setLast_update_time(System.currentTimeMillis() + "");
			System.out.println(StringUtil.objectToJson(weibo));
			blogList.add(weibo);
		}
	}

	/**
	 * 解析用户的地区简介等信息
	 * 
	 * @param html
	 */
	public void parseUserABInfo(String html) {
		Document userDocument = Jsoup.parse(html);
		Elements fixElements = userDocument
				.select("div.WB_cardwrap>div.PCD_person_info>div.WB_innerwrap>div.m_wrap>div.detail>ul.ul_detail>li");
		for (Element element2 : fixElements) {
			Element liTypeEle = element2.select("li>span>em").first();
			if (null == liTypeEle)
				continue;
			String span_classs = liTypeEle.attr("class");
			if (span_classs.contains("ficon_cd_place")) {
				Element areaE = element2.select("li>span.item_text").first();
				String area = StringUtil.element2text(areaE);
				userinfo.setArea(area);
			} else if (span_classs.contains("ficon_pinfo")) {
				Element briefE = element2.select("li>span.item_text").first();
				String brief = StringUtil.element2text(briefE);
				brief = brief.replaceAll("简介： ", "");
				System.err.println("sssss"+brief);
				userinfo.setBrief(brief);
			}
		}
	}

	/**
	 * 解析用户的关注数、粉丝数、微博数
	 * 
	 * @param html
	 */
	public void parseUserFansInfo(String html) {
		Document documentUs = Jsoup.parse(html);
		Elements element_fans_att_wb = null;
		element_fans_att_wb = documentUs
				.select("div.WB_cardwrap>div.PCD_counter>div.WB_innerwrap>table.tb_counter>tbody>tr>td>a>strong");
		if (element_fans_att_wb.size() < 1) {
			element_fans_att_wb = documentUs
					.select("div.WB_cardwrap>div.PCD_counter>div.WB_innerwrap>table.tb_counter>tbody>tr>td>strong");
		}
		String attenStr = StringUtil.element2text(element_fans_att_wb.get(0));
		String fansStr = StringUtil.element2text(element_fans_att_wb.get(1));
		String weiboStr = StringUtil.element2text(element_fans_att_wb.get(2));
		userinfo.setFollow_count(formatInteger(attenStr));
		userinfo.setFans_count(formatInteger(fansStr));
		userinfo.setWeibo_count(formatInteger(weiboStr));
	}

	/***
	 * 解析用户头像、性别、认证等信息
	 * 
	 * @param headerHtml
	 */
	public void parseUserHeaderInfo(String headerHtml) {
		Document headerDocument = Jsoup.parse(headerHtml);
		Element headerElement = headerDocument.select("div.PCD_header>div.pf_wrap>div.shadow").first();
		Element imgEle = headerElement.select("div>div.pf_photo>p.photo_wrap>img").first();
		String imgUrl = imgEle.attr("src");
		String nickname = imgEle.attr("alt");
		userinfo.setUser_img(imgUrl);
		userinfo.setNick_name(nickname);
		// 获取认证
		Element attn = headerElement.select("div>div.pf_photo>a>em").first();
		if (null != attn) {
			String atten_msg = attn.attr("title");
			String atten_clas = attn.attr("class");
			if (atten_clas.contains("gold")) {
				// 黄
				userinfo.setAuthentication_type(2);
			} else if (atten_clas.contains("approve_co")) {
				// 蓝
				userinfo.setAuthentication_type(1);
			}
			userinfo.setAuthentication_msg(atten_msg);
		}
		// 性别
		Element pf_usernameEle = headerDocument.select("div.PCD_header>div.pf_wrap>div.shadow>div.pf_username").first();
		Element genderEle = pf_usernameEle.select("div>span>a>i").first();
		int gender = 2;
		if (null != genderEle) {
			String ccGen = genderEle.attr("class");
			if (ccGen.contains("female"))
				gender = 0;
			else
				gender = 1;
		}
		userinfo.setUser_gender(gender);
		// 会员
		int member = 0;
		Element memberELe = pf_usernameEle.select("div.pf_username>a[action-type=ignore_list]").first();
		if (null != memberELe) {
			member = 1;
		}
		userinfo.setUser_member(member);
	}

	/**
	 * 格式化数字
	 * 
	 * @param num
	 * @return
	 */
	public int formatInteger(String num) {
		num = num.replaceAll(" ", "");
		num = num.replaceAll("	", "");
		num = num.replaceAll(" ", "");
		num = num.replaceAll("-", "");
		if (num.contains("万")) {
			num = num.replaceAll("万", "");
			return Integer.parseInt(num) * 10000;
		} else {
			return Integer.parseInt(num);
		}
	}

	/**
	 * 过滤字符
	 */
	public int getInteger(String num) {
		num = num.replaceAll("", "").replaceAll(" ", "").replaceAll("ñ", "");
		num = num.replaceAll("", "");
		if (StringUtil.isNullOrEmpty(num)) {
			return 0;
		} else {
			try {
				return Integer.parseInt(num);
			} catch (Exception e) {
				System.out.println("转换错误" + num);
				return 0;
			}
		}
	}

	/**
	 * 过滤URL
	 */
	public String filterUrl(String url) {
		if (url.contains("?")) {
			String[] surl = url.split("\\?");
			return surl[0];
		}
		return url;
	}

	/***
	 * 请求头封装在Map中，Cookie从本地文件读取
	 */
	public Map<String, String> requestHeader() {
		Map<String, String> requestHeader = new HashMap<String, String>();
		requestHeader.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		requestHeader.put("Accept-Encoding", "gzip, deflate, sdch");
		requestHeader.put("Accept-Language", "zh-CN,zh;q=0.8");
		requestHeader.put("Cache-Control", "max-age=0");
		requestHeader.put("Connection", "keep-alive");
		requestHeader.put("Cookie", FileUtil.txt2String(new File("C:/sinalogincookie.txt")));
		requestHeader.put("Host", "weibo.com");
		requestHeader.put("Upgrade-Insecure-Requests", "1");
		requestHeader.put("User-Agent",
				"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
		return requestHeader;
	}

}
